package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.AdjustIsNewProcessFunction2;
import net.bwie.realtime.jtp.dwd.log.function.AppLogSplitProcessFunction2;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

public class JtpLogEtlJob2 {
    public static void main(String[] args) throws Exception {
        //1.执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        //2.数据源-source
        DataStream<String> kafkaDateStream = KafkaUtil.consumerKafka(env, "topic-log");
//        kafkaDateStream.print("kafka);

        //3.数据转换-transformation
        DataStream<String> pageStream=processLog(kafkaDateStream);

        //4.数据输出-sink
        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");

        //5.执行-execute
        env.execute("JtpLogEtlJob2");
    }

    private static DataStream<String> processLog(DataStream<String> stream) {
        //1.数据清洗
        DataStream<String> jsonStream = appLogCleaned(stream);

        //2.新老访客状态标记修复
        DataStream<String> etlStream = processIsNew(jsonStream);

        //3.数据分流
        DataStream<String> pageStream = splitStream(jsonStream);

        return pageStream;
    }

    private static DataStream<String> splitStream(DataStream<String> stream) {
        //第一步：测边流输出标记
        OutputTag<String> errorTag = new OutputTag<String>("error-log") {};
        OutputTag<String> startTag = new OutputTag<String>("start-log") {};
        OutputTag<String> displaysTag = new OutputTag<String>("displays-log") {};
        OutputTag<String> actionTag = new OutputTag<String>("action-log") {};

        //第二步：日志分流处理
        SingleOutputStreamOperator<String> pageStram = stream.process(
                new AppLogSplitProcessFunction2(errorTag, startTag, displaysTag, actionTag));
        
        //第三步：测流输出
        SideOutputDataStream<String> errorStream = pageStram.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        SideOutputDataStream<String> startStream = pageStram.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        SideOutputDataStream<String> displaysStream = pageStram.getSideOutput(displaysTag);
        KafkaUtil.producerKafka(displaysStream,"dwd-traffic-displays-log");
        SideOutputDataStream<String> actionStream = pageStram.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");

        //第四步：输出主流
        return pageStram;

    }

    private static DataStream<String> processIsNew(DataStream<String> stream) {
        //a-按照设备ID进行分组
        KeyedStream<String, String> midStream = stream.keyBy(new KeySelector<String, String>() {
            @Override
            public String getKey(String s) throws Exception {
                return JSON.parseObject(s).getJSONObject("common").getString("mid");
            }
        });

        //b-状态编程，对is_new校验修复
        DataStream<String> isNewStream = midStream.process(new AdjustIsNewProcessFunction2());

        //c-返回数据流
        return isNewStream;
    }

    private static DataStream<String> appLogCleaned(DataStream<String> stream) {
        //a-脏数据侧边流输出时标签
        OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log") {};

        //b-数据清洗处理
        SingleOutputStreamOperator<String> cleanedStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, ProcessFunction<String, String>.Context ctx, Collector<String> out) throws Exception {
                try {
                    //a.解析JSON数据
                    JSON.parseObject(value);
                    //b.没有异常，没有解析，正常输出
                    out.collect(value);
                }catch (Exception e){
                    //c.捕获异常，侧边流输出数据
                    ctx.output(dirtyTag,value);
                }
            }
        });

        //c-侧边流输出：脏数据
        SideOutputDataStream<String> dirtyStream = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-log");

        //d-返回正常数据流
        return cleanedStream;
    }
}
